传统视觉方法思路:
首先对一种徽章,采样一批正确样本,用这些操作计算徽章模板图像:
对测试图片,同样计算并应用应用单应性矩阵进行透视变形,然后与模板图像计算结构相似度(structural similarity)。
采样一批验证集,统计结构相似度和准确率/召回率的关系。按进审率卡结构相似度,来决定一个是通过/抛弃还是进入人审
from fastai.vision.all import *
/home/tiger/.local/lib/python3.7/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:100.) return torch._C._cuda_getDeviceCount() > 0
fns = get_image_files('徽章的图像/老虎')
fns[-4:]
(#4) [Path('徽章的图像/老虎/良品/老虎不明显/Image_20201013165228873.bmp'),Path('徽章的图像/老虎/良品/老虎不明显/Image_20201013165242565.bmp'),Path('徽章的图像/老虎/良品/老虎不明显/Image_20201013165204095.bmp'),Path('徽章的图像/老虎/良品/老虎不明显/Image_20201013165255001.bmp')]
imgs_correct = [Image.open(fn) for fn in fns[-4:]]
imgs_correct_thumb = []
for img in imgs_correct:
thumb = img.copy()
thumb.thumbnail((512, 512))
imgs_correct_thumb.append(thumb)
as_correct_thumb = [np.array(o) for o in imgs_correct_thumb]
imgs_correct_thumb[0]
imgs_incorrect = [Image.open(fn) for fn in fns[:-4]]
imgs_incorrect_thumb = []
for img in imgs_incorrect:
thumb = img.copy()
thumb.thumbnail((512, 512))
imgs_incorrect_thumb.append(thumb)
as_incorrect_thumb = [np.array(o) for o in imgs_incorrect_thumb]
imgs_incorrect_thumb[0]
# import the necessary packages
import cv2
import numpy as np
def stitch(images, ratio=0.75, reprojThresh=4.0,
showMatches=False):
# unpack the images, then detect keypoints and extract
# local invariant descriptors from them
(imageB, imageA) = images
(kpsA, featuresA) = detectAndDescribe(imageA)
(kpsB, featuresB) = detectAndDescribe(imageB)
# match features between the two images
M = matchKeypoints(kpsA, kpsB,
featuresA, featuresB, ratio, reprojThresh)
# print(M)
# if the match is None, then there aren't enough matched
# keypoints to create a panorama
if M is None:
return None
# otherwise, apply a perspective warp to stitch the images
# together
(matches, H, status) = M
result = cv2.warpPerspective(imageA, H,
(imageA.shape[1],
imageA.shape[0]))
result[0:imageB.shape[0], 0:imageB.shape[1]] = result[0:imageB.shape[0], 0:imageB.shape[1]] // 2 + imageB // 2
# result = result // 2
# check to see if the keypoint matches should be visualized
if showMatches:
vis = drawMatches(imageA, imageB, kpsA, kpsB, matches,
status)
# return a tuple of the stitched image and the
# visualization
return (result, vis)
# return the stitched image
return result
def detectAndDescribe(image):
# convert the image to grayscale
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# detect and extract features from the image
descriptor = cv2.xfeatures2d.SIFT_create()
(kps, features) = descriptor.detectAndCompute(image, None)
# convert the keypoints from KeyPoint objects to NumPy
# arrays
kps = np.float32([kp.pt for kp in kps])
# return a tuple of keypoints and features
return (kps, features)
def matchKeypoints(kpsA, kpsB, featuresA, featuresB,
ratio, reprojThresh):
# compute the raw matches and initialize the list of actual
# matches
matcher = cv2.DescriptorMatcher_create("BruteForce")
rawMatches = matcher.knnMatch(featuresA, featuresB, 2)
matches = []
# loop over the raw matches
for m in rawMatches:
# ensure the distance is within a certain ratio of each
# other (i.e. Lowe's ratio test)
if len(m) == 2 and m[0].distance < m[1].distance * ratio:
matches.append((m[0].trainIdx, m[0].queryIdx))
# computing a homography requires at least 4 matches
if len(matches) > 4:
# construct the two sets of points
ptsA = np.float32([kpsA[i] for (_, i) in matches])
ptsB = np.float32([kpsB[i] for (i, _) in matches])
# compute the homography between the two sets of points
(H, status) = cv2.findHomography(ptsA, ptsB, cv2.RANSAC,
reprojThresh)
# return the matches along with the homograpy matrix
# and status of each matched point
return (matches, H, status)
# otherwise, no homograpy could be computed
return None
def drawMatches(imageA, imageB, kpsA, kpsB, matches, status):
# initialize the output visualization image
(hA, wA) = imageA.shape[:2]
(hB, wB) = imageB.shape[:2]
vis = np.zeros((max(hA, hB), wA + wB, 3), dtype="uint8")
vis[0:hA, 0:wA] = imageA
vis[0:hB, wA:] = imageB
# loop over the matches
for ((trainIdx, queryIdx), s) in zip(matches, status):
# only process the match if the keypoint was successfully
# matched
if s == 1:
# draw the match
ptA = (int(kpsA[queryIdx][0]), int(kpsA[queryIdx][1]))
ptB = (int(kpsB[trainIdx][0]) + wA, int(kpsB[trainIdx][1]))
cv2.line(vis, ptA, ptB, (0, 255, 0), 1)
# return the visualization
return vis
a_stitched = as_correct_thumb[0]
for a in as_correct_thumb[1:-1]:
a_stitched, vis = stitch((a_stitched, a), reprojThresh=6, showMatches=True)
img_stitched = Image.fromarray(a_stitched)
img_vis = Image.fromarray(vis)
display(img_stitched, img_vis)
def warp(imageA, imageB, ratio=0.75, reprojThresh=6):
(kpsA, featuresA) = detectAndDescribe(imageA)
(kpsB, featuresB) = detectAndDescribe(imageB)
# match features between the two images
M = matchKeypoints(kpsA, kpsB,
featuresA, featuresB, ratio, reprojThresh)
# print(M)
# if the match is None, then there aren't enough matched
# keypoints to create a panorama
if M is None:
return None
# otherwise, apply a perspective warp to stitch the images
# together
(matches, H, status) = M
warpped = cv2.warpPerspective(imageA, H,
(imageA.shape[1],
imageA.shape[0]))
# print(np.sum(warpped - imageA))
return warpped
import skimage.metrics
a_stitched = as_correct_thumb[0]
warpped = [a_stitched]
for a in as_correct_thumb[1:]:
warpped.append(warp(a, a_stitched))
a_stitched = np.mean(warpped, 0).astype(np.uint8)
Image.fromarray(a_stitched)
error_pairs = []
for a in as_correct_thumb:
warpped = warp(a, a_stitched, reprojThresh=4)
if warpped is None:
display(Image.fromarray(a), None)
continue
img = Image.fromarray(warpped)
score = skimage.metrics.structural_similarity(warpped, a_stitched, multichannel=True)
error_pairs.append((img, score))
# display(*error_pairs[-1])
error_pairs.sort(key=lambda x: x[1])
for p in error_pairs:
if True: #p[1] > 0.8:
img, score = p
diff = Image.fromarray(np.abs(np.array(img).astype(np.float64) - a_stitched.astype(np.float64)).astype(np.uint8))
fig, ax = plt.subplots(1, 2, figsize=(20, 10))
ax[0].imshow(img)
plt.axis('off')
ax[1].imshow(diff)
plt.title(f'结构相似度:{score}')
plt.axis('off')
error_pairs = []
for a in as_incorrect_thumb:
warpped = warp(a, a_stitched, reprojThresh=4)
if warpped is None:
display(Image.fromarray(a), None)
continue
img = Image.fromarray(warpped)
score = skimage.metrics.structural_similarity(warpped, a_stitched, multichannel=True)
error_pairs.append((img, score))
# display(*error_pairs[-1])
error_pairs.sort(key=lambda x: x[1])
for p in error_pairs[::5]:
if True: #p[1] > 0.8:
img, score = p
diff = Image.fromarray(np.abs(np.array(img).astype(np.float64) - a_stitched.astype(np.float64)).astype(np.uint8))
fig, ax = plt.subplots(1, 2, figsize=(20, 10))
ax[0].imshow(img)
ax[0].set_axis_off()
ax[1].imshow(diff)
ax[1].set_axis_off()
plt.title(f'结构相似度:{score}')
/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).